package edu.hawaii.webspider;

import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;

import com.meterware.httpunit.GetMethodWebRequest;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebLink;
import com.meterware.httpunit.WebRequest;
import com.meterware.httpunit.WebResponse;

/**
 * A demonstration class illustrating how to retrieve and process web pages using HttpUnit.
 * 
 * @author Philip Johnson
 * @author Jianfei Liao
 */
public class WebSpiderExample {

  /** Contains the starting url for the crawl. */
  private final String startUrl;

  /** Total number of links. */
  private int numTotalLinks = 0;

  /** New links to visit. */
  private final List<String> newLinks = new ArrayList<String>();

  /** Links already visited. */
  private final List<String> visitedLinks = new ArrayList<String>();

  /** Number of pages to be checked. */
  private int numPagesLimit = 0;
  
  /** Links already visited with counters. */
  private final List<WebItem> counters = new ArrayList<WebItem>();
  
  /** Logger for the program. */
  private final Logger log = Logger.getLogger("edu.hawaii.webspider");
  
  /** Logging state. */
  private boolean doLogging = false;

  /**
   * A WebSpider crawls the web and returns info.
   * 
   * @param startUrl The home url for the crawl.
   */
  public WebSpiderExample(String startUrl) {
    this.startUrl = startUrl;
  }

  /**
   * Returns the number of links found at the startUrl.
   * 
   * @return The number of links.
   * @throws Exception if problems occur retrieving the startUrl.
   */
  public int getNumLinks() throws Exception {
    // create the conversation object which will maintain state for us
    WebConversation wc = new WebConversation();
    WebRequest request = new GetMethodWebRequest(this.startUrl);
    WebResponse response = wc.getResponse(request);
    return response.getLinks().length;
  }

  /**
   * Retrieves the httpunit home page and counts the number of links on it.
   * 
   * @param args Ignored.
   * @throws Exception If problems occur.
   */
  public static void main(String[] args) throws Exception {
    // check the arguments and perform different tasks
    int numArgs = args.length;
    if (numArgs < 3) {
      System.out.println("3 arguments are expected.");
    }
    else {
      // create a new instance of WebSpider
      WebSpiderExample example = new WebSpiderExample(args[1]);
      // create a logger
      Logger log = Logger.getLogger("edu.hawaii.spider");
      boolean doLog = true;
      
      // convert the String argument to an integer
      int limit = 0;
      try {
        limit = Integer.parseInt(args[2]);
      }
      catch (Exception e) {
        if (doLog) {
          log.warning("Argument for number of pages to crawl is not an integer.");
        }
        else {
          System.out.println("Exception during initializing the arguments: " + e.getMessage());
        }
      }
      // if logging is not enabled
      if (numArgs == 3 && args[0].equalsIgnoreCase("-totallinks")) {
        // perform the total links task without logging
        int totalLinks = example.findTotalLinks(limit, false);
        System.out.println("The total number of links discovered while crawling the first "
            + limit + " pages accessible from " + args[1] + " is: " + totalLinks);
      }
      else if (numArgs == 3 && args[0].equalsIgnoreCase("-mostpopular")) {
        // perform the most popular task without logging
        WebItem item = example.findMostPopular(limit, false);
        System.out.println("The most popular page discovered is: " + item.getLink() + " with " 
             + item.getCounter() + " pages that point to it");
      }
      // if logging is enabled
      else if (numArgs == 4 && args[0].equalsIgnoreCase("-totallinks")
          && args[3].equalsIgnoreCase("-logging")) {
        // perform the total links task with logging
        int totalLinks = example.findTotalLinks(limit, true);
        System.out.println("The total number of links discovered while crawling the first "
            + limit + " pages accessible from " + args[1] + " is: " + totalLinks); 
      }
      else if (args[0].equalsIgnoreCase("-mostpopular") && args[3].equalsIgnoreCase("-logging")) {
        // perform the most popular task with logging
        WebItem item = example.findMostPopular(limit, true);
        System.out.println("The most popular page discovered is: " + item.getLink() + " with " 
             + item.getCounter() + " pages that point to it");
      }
      else {
        System.out.println("Arguments passed incorrectly.");
      }
    }
  }
  
  /**
   * Finds the total number of links discovered while crawling the first <code>limit</code> pages
   * accessible from <code>startUrl</code>. This method is heavily inspired by Jared Sunouchi and
   * Kevin English's codes.
   * 
   * @param limit The limit number of pages to crawl.
   * @param loggingState Do logging or not.
   * @return The total number of links discovered.
   * @throws Exception If errors happen during the crawling.
   */
  public int findTotalLinks(int limit, boolean loggingState) throws Exception {
    doLogging = loggingState;
    if (doLogging) {
      log.info("Start finding total number of links...");
    }
    
    // These 3 lines codes are inspired by Kevin English
    
    // prevent javascript errors from being thrown
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);

    // The following code is heavily inspired by Jared Sunouchi

    // set up variables
    numPagesLimit = limit;
    String curUrl = this.startUrl;
    // add the start URL to newLinks
    newLinks.add(curUrl);
    while (!newLinks.isEmpty() && numPagesLimit > 0) {
      curUrl = newLinks.remove(0);
      if (doLogging) {
        log.info("Retrieving URL: " + curUrl 
            + "\nCurrent total links: " + numTotalLinks
            + "\nCurrent visited pages: " + (100 - numPagesLimit + 1) + "\n");
      }
      try {
        // if the curUrl is not visited, get all links from it
        if (!visitedLinks.contains(curUrl)) {
          WebConversation wc = new WebConversation();
          WebResponse response = wc.getResponse(curUrl);
          numPagesLimit--;
          visitedLinks.add(curUrl);
          WebLink[] webLinksArray = response.getLinks();

          // store the links on curUrl to newLinks, and add up the number of links in it
          for (WebLink curWebLink : webLinksArray) {
            curUrl = curWebLink.getRequest().getURL().toString();
            if (!newLinks.contains(curUrl)) {
              newLinks.add(curUrl);
            }
          }
          numTotalLinks = numTotalLinks + webLinksArray.length;
        }
      }
      catch (Exception e) {
        if (doLogging) {
          log.warning("Thrown exception: " + e.getMessage());
        }
        else {
          System.out.println("Exception during the total links task: " + e.getMessage());
        }
      }  
    }
    return numTotalLinks;
  }
  
  /**
   * Finds the most popular page with most links point back to it. This method is inspired by 
   * Ka Yee Leung's and Kevin English's code.
   * 
   * @author Jianfei Liao
   * 
   * @param limit The limit number of pages to crawl.
   * @param loggingState Do logging or not.
   * @return WebItem  A class with the link URL in String format and its counter.
   * @throws Exception If errors occur during the crawling. 
   */
  public WebItem findMostPopular(int limit, boolean loggingState) throws Exception {
    doLogging = loggingState;
    // These 3 lines codes are inspired by Kevin English
    
    // prevent javascript errors from being thrown
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnScriptError(false);
    com.meterware.httpunit.HttpUnitOptions.setExceptionsThrownOnErrorStatus(false);
    com.meterware.httpunit.HttpUnitOptions.setScriptingEnabled(false);
    
    // set up variables
    numPagesLimit = limit;
    String curUrl = this.startUrl;
    // add the start URL to newLinks
    newLinks.add(curUrl);
    while (!newLinks.isEmpty() && numPagesLimit > 0) {
      curUrl = newLinks.remove(0);
      if (doLogging) {
        log.info("Retrieving URL: " + curUrl 
            + "\nCurrent visited pages: " + (100 - numPagesLimit + 1) + "\n");
      }
      try {
        // if the curUrl has been visited, increase its counter
        if (visitedLinks.contains(curUrl)) {
          for (WebItem item : counters) {
            if (item.getLink().equalsIgnoreCase(curUrl)) {
              item.increaseCounter();
            }
          }          
        }
        // else the curUrl is not visited, get all links from it
        else {
          WebConversation wc = new WebConversation();
          WebResponse response = wc.getResponse(curUrl);
          numPagesLimit--;
          visitedLinks.add(curUrl);
          counters.add(new WebItem(curUrl));
          
          WebLink[] webLinksArray = response.getLinks();

          // store the links on curUrl to newLinks, and add up the number of links in it
          for (WebLink curWebLink : webLinksArray) {
            curUrl = curWebLink.getRequest().getURL().toString();
            newLinks.add(curUrl);
          }
        }
      }
      catch (Exception e) {
        if (doLogging) {
          log.warning("Thrown exception: " + e.getMessage());
        }
        else {
          System.out.println("Exception during most popular task: " + e.getMessage());
        }
      }
    }
    
    // find most popular
    int tempMostPopular = 0;
    WebItem tempItem = new WebItem("");
    for (WebItem item : counters) {
      if (item.getCounter() >= tempMostPopular) {
        tempMostPopular = item.getCounter();
        tempItem = item;
      }
    }
    return tempItem;
  }

  /**
   * Stores a link and a counter for the links point to it.
   * This is inspired by Ka Yee Leung's code.
   * @author Jianfei Liao
   */
  static class WebItem {
    /** A link and a counter. */
    protected final String link;
    protected int counter;
    
    /**
     *  Constructor.
     *  
     *  @param link The link to store.
     */
    public WebItem(String link) {
      this.link = link;
      this.counter = 1;
    }
    
    /** 
     * Accessor method for getting the link.
     * 
     * @return String The URL of this link in String form.
     */
    public String getLink() {
      return this.link;
    }
    
    /**
     * Accessor method for getting the counter.
     *
     * @return int The number of count of this link.
     */
    public int getCounter() {
      return this.counter;
    }
    
    /**
     *  Increase the counter of the item by one.
     *  
     */
    public void increaseCounter() {
      this.counter++;
    }
  }
}
